# 使用线程的方式在本地写入图片信息

import requests
from lxml import etree
from concurrent.futures import ThreadPoolExecutor, as_completed
import os
import time

url = 'https://www.lpbzj.vip/allimg/page/{}'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
    'Referer': 'https://www.lpbzj.vip/allimg'
}


# 获取详情页的url
def get_detail_url(page):
    response = requests.get(url.format(page), headers=headers)
    html = etree.HTML(response.text)
    detail_url_list = html.xpath("//div[@class='img']/a/@href")
    return detail_url_list


# 获取图片的url
def get_img_url(detail_url):
    response = requests.get(detail_url, headers=headers)
    if response.status_code == 200:
        html = etree.HTML(response.text)
        img_url_list = html.xpath("//div[@class='article-content clearfix']/p/img/@src")
        print(f'{detail_url}请求成功！')
        # with ThreadPoolExecutor() as inner_pool:
        #     inner_pool.map(download_url, img_url_list)
        return img_url_list
    else:
        print(f'{detail_url}请求失败,资源不存在！')


def download_url(url):
    response = requests.get(url, headers=headers).content
    img_name = url.split('/')[-1]
    with open(f'./img/{img_name}', 'wb') as f:
        f.write(response)
    print(f'{url}下载完成')


if __name__ == '__main__':
    if not os.path.exists('./img'):
        os.mkdir('./img')
    start_time = time.time()
    with ThreadPoolExecutor() as executor:
        # futures = [pool.submit(get_detail_url, page) for page in range(1, 2)]
        # for future in as_completed(futures):
        #     pool.map(get_img_url, future.result())
        # 第一阶段：获取所有分页的详情页URL
        page_futures = [executor.submit(get_detail_url, page) for page in range(1, 17)]

        # 第二阶段：处理每个详情页
        detail_futures = []
        for page_future in as_completed(page_futures):
            for detail_url in page_future.result():
                detail_futures.append(executor.submit(get_img_url, detail_url))

        # 第三阶段：下载所有图片
        for detail_future in as_completed(detail_futures):
            for img_url in detail_future.result():
                executor.submit(download_url, img_url)
    end_time = time.time()
    print(f'总体耗时：{time.time() - start_time}')
